Non fxa migrated deletions

kitsune/users/migrations/0033_batch_delete_non_migrated_users.py

@@ -0,0 +1,120 @@
+import time
+from datetime import timedelta
+from itertools import islice
+
+from django.db import migrations
+from django.db.models import Q
+
+
+def delete_non_migrated_users(apps, schema_editor):
+    """
+    Delete users where is_fxa_migrated is False and who aren't creators/owners/users of
+    any content.
+    """
+    User = apps.get_model("auth", "User")
+    users_to_delete = User.objects.filter(profile__is_fxa_migrated=False).exclude(
+        Q(answer_votes__isnull=False)
+        | Q(answers__isnull=False)
+        | Q(award_creator__isnull=False)
+        | Q(badge__isnull=False)
+        | Q(created_revisions__isnull=False)
+        | Q(gallery_images__isnull=False)
+        | Q(gallery_videos__isnull=False)
+        | Q(inboxmessage__isnull=False)
+        | Q(outbox__isnull=False)
+        | Q(poll_votes__isnull=False)
+        | Q(post__isnull=False)
+        | Q(question_votes__isnull=False)
+        | Q(questions__isnull=False)
+        | Q(readied_for_l10n_revisions__isnull=False)
+        | Q(reviewed_revisions__isnull=False)
+        | Q(thread__isnull=False)
+        | Q(wiki_post_set__isnull=False)
+        | Q(wiki_thread_set__isnull=False)
+        | Q(locales_leader__isnull=False)
+        | Q(locales_reviewer__isnull=False)
+        | Q(locales_editor__isnull=False)
+        | Q(wiki_contributions__isnull=False)
+    )
+
+    total_users = users_to_delete.count()
+    if total_users == 0:
+        print("No users to delete")
+        return
+
+    print(f"Starting deletion of {total_users:,} users")
+    start_time = time.time()
+    deleted_count = 0
+    batch_size = 2000
+
+    user_ids = users_to_delete.values_list("id", flat=True).iterator(chunk_size=batch_size)
+    current_batch = []
+
+    for user_id in user_ids:
+        current_batch.append(user_id)
+
+        if len(current_batch) >= batch_size:
+            # Delete the batch using _base_manager to avoid the overridden managers
+            # of each model through the cascade.
+            # We don't care about the extra logic b/c the accounts that are being deleted are empty
+            deletion_counts = User._base_manager.filter(id__in=current_batch).delete()
+            # get the user, not the cascaded deletes
+            user_deletes = deletion_counts[1].get("auth.User", 0)
+            deleted_count += user_deletes
+            current_batch = []
+
+            elapsed_time = time.time() - start_time
+            avg_time_per_user = elapsed_time / deleted_count if deleted_count > 0 else 0
+            current_rate = deleted_count / elapsed_time * 60 if elapsed_time > 0 else 0
+            remaining_time = (
+                (total_users - deleted_count) * avg_time_per_user if deleted_count > 0 else 0
+            )
+
+            print(
+                f"""
+                Progress Report:
+                ---------------
+                Users Deleted: {deleted_count:,} of {total_users:,} ({(deleted_count/total_users*100):.1f}%)
+                Elapsed Time: {timedelta(seconds=int(elapsed_time))}
+                Average Time per User: {avg_time_per_user:.3f} seconds
+                Current Rate: {current_rate:.1f} users/minute
+                Estimated Time Remaining: {timedelta(seconds=int(remaining_time))}
+                """
+            )
+
+    if current_batch:
+        deletion_counts = User._base_manager.filter(id__in=current_batch).delete()
+        user_deletes = deletion_counts[1].get("auth.User", 0)
+        deleted_count += user_deletes
+
+    total_time = time.time() - start_time
+    print(
+        f"""
+        Deletion Complete:
+        -----------------
+        Total Users Deleted: {deleted_count:,} of {total_users:,}
+        Total Time: {timedelta(seconds=int(total_time))}
+        Average Time per User: {(total_time/deleted_count if deleted_count > 0 else 0):.3f} seconds
+        Overall Rate: {(deleted_count/total_time*60 if total_time > 0 else 0):.1f} users/minute
+        """
+    )
+
+
+def reverse_migration(apps, schema_editor):
+    """
+    No reverse migration possible since deletion cannot be undone
+    """
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("users", "0032_profile_account_type_alter_profile_user"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            delete_non_migrated_users,
+            reverse_migration,
+        ),
+    ]

kitsune/users/tests/test_migration_0033.py

@@ -0,0 +1,110 @@
+from django.contrib.auth.models import User
+from django.test import TestCase
+from django.db.models import Q
+
+from kitsune.questions.tests import (
+    AnswerFactory,
+    QuestionFactory,
+    QuestionVoteFactory,
+    AnswerVoteFactory,
+)
+from kitsune.users.tests import ProfileFactory
+from kitsune.wiki.tests import RevisionFactory
+from kitsune.messages.models import InboxMessage, OutboxMessage
+from kitsune.gallery.tests import ImageFactory, VideoFactory
+from kitsune.forums.tests import ThreadFactory, PostFactory
+
+
+class TestDeleteNonMigratedUsersMigrationQuery(TestCase):
+    """Test the migration that deletes non-migrated users."""
+
+    def setUp(self):
+        # Create users with different combinations of migration status and content creation
+
+        # Case 1: Non-migrated user with no content (should be deleted)
+        ProfileFactory(user__username="user1", is_fxa_migrated=False)
+
+        # Case 2: Non-migrated user with a question (should be kept)
+        user2 = ProfileFactory(user__username="user2", is_fxa_migrated=False).user
+        QuestionFactory(creator=user2)
+
+        # Case 3: Non-migrated user with an answer (should be kept)
+        user3 = ProfileFactory(user__username="user3", is_fxa_migrated=False).user
+        AnswerFactory(creator=user3)
+
+        # Case 4: Non-migrated user with a revision (should be kept)
+        user4 = ProfileFactory(user__username="user4", is_fxa_migrated=False).user
+        RevisionFactory(creator=user4)
+
+        # Case 5: Migrated user with no content (should be kept)
+        ProfileFactory(user__username="user5", is_fxa_migrated=True)
+
+        # Case 6: Non-migrated user with a question vote (should be kept)
+        user6 = ProfileFactory(user__username="user6", is_fxa_migrated=False).user
+        QuestionVoteFactory(creator=user6)
+
+        # Case 7: Non-migrated user with an answer vote (should be kept)
+        user7 = ProfileFactory(user__username="user7", is_fxa_migrated=False).user
+        AnswerVoteFactory(creator=user7)
+
+        # Case 8: Non-migrated user who is a sender of inbox messages (should be kept)
+        user8 = ProfileFactory(user__username="user8", is_fxa_migrated=False).user
+        InboxMessage.objects.create(to=user2, sender=user8, message="test")
+
+        # Case 9: Non-migrated user with an outbox message (should be kept)
+        user9 = ProfileFactory(user__username="user9", is_fxa_migrated=False).user
+        outbox_msg = OutboxMessage.objects.create(sender=user9, message="test")
+        outbox_msg.to.add(user2)
+
+        # Case 10: Non-migrated user who is a sender of inbox messages (should be kept)
+        user10 = ProfileFactory(user__username="user10", is_fxa_migrated=False).user
+        InboxMessage.objects.create(to=user2, sender=user10, message="test")
+
+        # Case 11: Non-migrated user with a gallery image (should be kept)
+        user11 = ProfileFactory(user__username="user11", is_fxa_migrated=False).user
+        ImageFactory(creator=user11)
+
+        # Case 12: Non-migrated user with a gallery video (should be kept)
+        user12 = ProfileFactory(user__username="user12", is_fxa_migrated=False).user
+        VideoFactory(creator=user12)
+
+        # Case 13: Non-migrated user with a forum thread (should be kept)
+        user13 = ProfileFactory(user__username="user13", is_fxa_migrated=False).user
+        ThreadFactory(creator=user13)
+
+        # Case 14: Non-migrated user with a forum post (should be kept)
+        user14 = ProfileFactory(user__username="user14", is_fxa_migrated=False).user
+        thread = ThreadFactory()
+        PostFactory(thread=thread, author=user14)
+
+        # Case 15: Non-migrated user who reviewed a revision (should be kept)
+        user15 = ProfileFactory(user__username="user15", is_fxa_migrated=False).user
+        RevisionFactory(reviewer=user15)
+
+    def test_direct_query_logic(self):
+        """Test the query logic of the migration directly without going through apps."""
+        # Query using the same logic as the migration but with direct model references
+        users_to_delete = User.objects.filter(profile__is_fxa_migrated=False).exclude(
+            Q(answer_votes__isnull=False)
+            | Q(answers__isnull=False)
+            | Q(award_creator__isnull=False)
+            | Q(badge__isnull=False)
+            | Q(created_revisions__isnull=False)
+            | Q(gallery_images__isnull=False)
+            | Q(gallery_videos__isnull=False)
+            | Q(inboxmessage__isnull=False)
+            | Q(outbox__isnull=False)
+            | Q(poll_votes__isnull=False)
+            | Q(post__isnull=False)
+            | Q(question_votes__isnull=False)
+            | Q(questions__isnull=False)
+            | Q(readied_for_l10n_revisions__isnull=False)
+            | Q(reviewed_revisions__isnull=False)
+            | Q(thread__isnull=False)
+            | Q(wiki_post_set__isnull=False)
+            | Q(wiki_thread_set__isnull=False)
+        )
+
+        # Only user1 should be in this queryset
+        self.assertEqual(users_to_delete.count(), 1)
+        self.assertEqual(users_to_delete.first().username, "user1")